{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\software\\envs\\tensorflow\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-e49e04966a9d>:69: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is deprecated, please use tf.nn.rnn_cell.LSTMCell, which supports all the feature this cell currently has. Please replace the existing code with tf.nn.rnn_cell.LSTMCell(name='basic_lstm_cell').\n",
      "WARNING:tensorflow:From <ipython-input-1-e49e04966a9d>:89: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "step0:\n",
      "Train accuracy:  0.1966\n",
      "Test accuracy:  0.1982\n",
      "step10:\n",
      "Train accuracy:  0.6788\n",
      "Test accuracy:  0.682\n",
      "step20:\n",
      "Train accuracy:  0.8354\n",
      "Test accuracy:  0.8316\n",
      "step30:\n",
      "Train accuracy:  0.8426\n",
      "Test accuracy:  0.8476\n",
      "step40:\n",
      "Train accuracy:  0.8436\n",
      "Test accuracy:  0.841\n",
      "step50:\n",
      "Train accuracy:  0.8744\n",
      "Test accuracy:  0.8766\n",
      "step60:\n",
      "Train accuracy:  0.8664\n",
      "Test accuracy:  0.862\n",
      "step70:\n",
      "Train accuracy:  0.8892\n",
      "Test accuracy:  0.8954\n",
      "step80:\n",
      "Train accuracy:  0.8822\n",
      "Test accuracy:  0.8814\n",
      "step90:\n",
      "Train accuracy:  0.9044\n",
      "Test accuracy:  0.903\n",
      "step100:\n",
      "Train accuracy:  0.898\n",
      "Test accuracy:  0.8992\n",
      "step110:\n",
      "Train accuracy:  0.9052\n",
      "Test accuracy:  0.9064\n",
      "step120:\n",
      "Train accuracy:  0.8994\n",
      "Test accuracy:  0.9034\n",
      "step130:\n",
      "Train accuracy:  0.9032\n",
      "Test accuracy:  0.9008\n",
      "step140:\n",
      "Train accuracy:  0.899\n",
      "Test accuracy:  0.9034\n",
      "step150:\n",
      "Train accuracy:  0.9038\n",
      "Test accuracy:  0.9026\n",
      "step160:\n",
      "Train accuracy:  0.9036\n",
      "Test accuracy:  0.9082\n",
      "step170:\n",
      "Train accuracy:  0.909\n",
      "Test accuracy:  0.9104\n",
      "step180:\n",
      "Train accuracy:  0.9004\n",
      "Test accuracy:  0.9056\n",
      "step190:\n",
      "Train accuracy:  0.9092\n",
      "Test accuracy:  0.9044\n",
      "step200:\n",
      "Train accuracy:  0.905\n",
      "Test accuracy:  0.9018\n",
      "step210:\n",
      "Train accuracy:  0.91\n",
      "Test accuracy:  0.9056\n",
      "step220:\n",
      "Train accuracy:  0.892\n",
      "Test accuracy:  0.8946\n",
      "step230:\n",
      "Train accuracy:  0.905\n",
      "Test accuracy:  0.9032\n",
      "step240:\n",
      "Train accuracy:  0.914\n",
      "Test accuracy:  0.9152\n",
      "step250:\n",
      "Train accuracy:  0.9004\n",
      "Test accuracy:  0.9\n",
      "step260:\n",
      "Train accuracy:  0.9124\n",
      "Test accuracy:  0.9104\n",
      "step270:\n",
      "Train accuracy:  0.9006\n",
      "Test accuracy:  0.901\n",
      "step280:\n",
      "Train accuracy:  0.9124\n",
      "Test accuracy:  0.8974\n",
      "step290:\n",
      "Train accuracy:  0.917\n",
      "Test accuracy:  0.9178\n",
      "step300:\n",
      "Train accuracy:  0.9002\n",
      "Test accuracy:  0.9034\n",
      "step310:\n",
      "Train accuracy:  0.9112\n",
      "Test accuracy:  0.9092\n",
      "step320:\n",
      "Train accuracy:  0.911\n",
      "Test accuracy:  0.9102\n",
      "step330:\n",
      "Train accuracy:  0.9158\n",
      "Test accuracy:  0.9182\n",
      "step340:\n",
      "Train accuracy:  0.9004\n",
      "Test accuracy:  0.8978\n",
      "step350:\n",
      "Train accuracy:  0.8972\n",
      "Test accuracy:  0.9054\n",
      "step360:\n",
      "Train accuracy:  0.9074\n",
      "Test accuracy:  0.9008\n",
      "step370:\n",
      "Train accuracy:  0.9066\n",
      "Test accuracy:  0.9044\n",
      "step380:\n",
      "Train accuracy:  0.916\n",
      "Test accuracy:  0.9154\n",
      "step390:\n",
      "Train accuracy:  0.9182\n",
      "Test accuracy:  0.9178\n",
      "step400:\n",
      "Train accuracy:  0.9152\n",
      "Test accuracy:  0.904\n",
      "step410:\n",
      "Train accuracy:  0.907\n",
      "Test accuracy:  0.9118\n",
      "step420:\n",
      "Train accuracy:  0.9078\n",
      "Test accuracy:  0.9088\n",
      "step430:\n",
      "Train accuracy:  0.8916\n",
      "Test accuracy:  0.902\n",
      "step440:\n",
      "Train accuracy:  0.9114\n",
      "Test accuracy:  0.9104\n",
      "step450:\n",
      "Train accuracy:  0.9194\n",
      "Test accuracy:  0.9156\n",
      "step460:\n",
      "Train accuracy:  0.9096\n",
      "Test accuracy:  0.9084\n",
      "step470:\n",
      "Train accuracy:  0.9138\n",
      "Test accuracy:  0.9126\n",
      "step480:\n",
      "Train accuracy:  0.9074\n",
      "Test accuracy:  0.9056\n",
      "step490:\n",
      "Train accuracy:  0.92\n",
      "Test accuracy:  0.9184\n",
      "step500:\n",
      "Train accuracy:  0.9112\n",
      "Test accuracy:  0.9072\n",
      "step510:\n",
      "Train accuracy:  0.9028\n",
      "Test accuracy:  0.9064\n",
      "step520:\n",
      "Train accuracy:  0.9046\n",
      "Test accuracy:  0.911\n",
      "step530:\n",
      "Train accuracy:  0.9164\n",
      "Test accuracy:  0.91\n",
      "step540:\n",
      "Train accuracy:  0.9022\n",
      "Test accuracy:  0.9052\n",
      "step550:\n",
      "Train accuracy:  0.9174\n",
      "Test accuracy:  0.9162\n",
      "step560:\n",
      "Train accuracy:  0.9154\n",
      "Test accuracy:  0.912\n",
      "step570:\n",
      "Train accuracy:  0.9064\n",
      "Test accuracy:  0.9072\n",
      "step580:\n",
      "Train accuracy:  0.907\n",
      "Test accuracy:  0.8992\n",
      "step590:\n",
      "Train accuracy:  0.9222\n",
      "Test accuracy:  0.9184\n",
      "step600:\n",
      "Train accuracy:  0.9048\n",
      "Test accuracy:  0.912\n",
      "step610:\n",
      "Train accuracy:  0.9132\n",
      "Test accuracy:  0.9074\n",
      "step620:\n",
      "Train accuracy:  0.9188\n",
      "Test accuracy:  0.9178\n",
      "step630:\n",
      "Train accuracy:  0.9134\n",
      "Test accuracy:  0.908\n",
      "step640:\n",
      "Train accuracy:  0.9154\n",
      "Test accuracy:  0.9146\n",
      "step650:\n",
      "Train accuracy:  0.9096\n",
      "Test accuracy:  0.9106\n",
      "step660:\n",
      "Train accuracy:  0.9188\n",
      "Test accuracy:  0.9146\n",
      "step670:\n",
      "Train accuracy:  0.909\n",
      "Test accuracy:  0.9076\n",
      "step680:\n",
      "Train accuracy:  0.9048\n",
      "Test accuracy:  0.9006\n",
      "step690:\n",
      "Train accuracy:  0.907\n",
      "Test accuracy:  0.9102\n",
      "step700:\n",
      "Train accuracy:  0.9052\n",
      "Test accuracy:  0.9038\n",
      "step710:\n",
      "Train accuracy:  0.9048\n",
      "Test accuracy:  0.9114\n",
      "step720:\n",
      "Train accuracy:  0.921\n",
      "Test accuracy:  0.915\n",
      "step730:\n",
      "Train accuracy:  0.9082\n",
      "Test accuracy:  0.9128\n",
      "step740:\n",
      "Train accuracy:  0.9208\n",
      "Test accuracy:  0.9182\n",
      "step750:\n",
      "Train accuracy:  0.9148\n",
      "Test accuracy:  0.912\n",
      "step760:\n",
      "Train accuracy:  0.9224\n",
      "Test accuracy:  0.9162\n",
      "step770:\n",
      "Train accuracy:  0.915\n",
      "Test accuracy:  0.9134\n",
      "step780:\n",
      "Train accuracy:  0.9198\n",
      "Test accuracy:  0.9226\n",
      "step790:\n",
      "Train accuracy:  0.9108\n",
      "Test accuracy:  0.905\n",
      "step800:\n",
      "Train accuracy:  0.916\n",
      "Test accuracy:  0.9224\n",
      "step810:\n",
      "Train accuracy:  0.9142\n",
      "Test accuracy:  0.9156\n",
      "step820:\n",
      "Train accuracy:  0.9064\n",
      "Test accuracy:  0.9104\n",
      "step830:\n",
      "Train accuracy:  0.9114\n",
      "Test accuracy:  0.9098\n",
      "step840:\n",
      "Train accuracy:  0.903\n",
      "Test accuracy:  0.9016\n",
      "step850:\n",
      "Train accuracy:  0.9246\n",
      "Test accuracy:  0.91\n",
      "step860:\n",
      "Train accuracy:  0.906\n",
      "Test accuracy:  0.9118\n",
      "step870:\n",
      "Train accuracy:  0.9144\n",
      "Test accuracy:  0.9154\n",
      "step880:\n",
      "Train accuracy:  0.9074\n",
      "Test accuracy:  0.9104\n",
      "step890:\n",
      "Train accuracy:  0.9164\n",
      "Test accuracy:  0.9118\n",
      "step900:\n",
      "Train accuracy:  0.9176\n",
      "Test accuracy:  0.9214\n",
      "step910:\n",
      "Train accuracy:  0.9006\n",
      "Test accuracy:  0.91\n",
      "step920:\n",
      "Train accuracy:  0.9242\n",
      "Test accuracy:  0.9176\n",
      "step930:\n",
      "Train accuracy:  0.923\n",
      "Test accuracy:  0.9166\n",
      "step940:\n",
      "Train accuracy:  0.922\n",
      "Test accuracy:  0.9194\n",
      "step950:\n",
      "Train accuracy:  0.9174\n",
      "Test accuracy:  0.9126\n",
      "step960:\n",
      "Train accuracy:  0.9064\n",
      "Test accuracy:  0.9092\n",
      "step970:\n",
      "Train accuracy:  0.9224\n",
      "Test accuracy:  0.912\n",
      "step980:\n",
      "Train accuracy:  0.9144\n",
      "Test accuracy:  0.9148\n",
      "step990:\n",
      "Train accuracy:  0.9122\n",
      "Test accuracy:  0.908\n",
      "step1000:\n",
      "Train accuracy:  0.9024\n",
      "Test accuracy:  0.905\n",
      "step1010:\n",
      "Train accuracy:  0.9174\n",
      "Test accuracy:  0.9186\n",
      "step1020:\n",
      "Train accuracy:  0.9188\n",
      "Test accuracy:  0.9182\n",
      "step1030:\n",
      "Train accuracy:  0.9184\n",
      "Test accuracy:  0.9112\n",
      "step1040:\n",
      "Train accuracy:  0.9204\n",
      "Test accuracy:  0.9208\n",
      "step1050:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.9188\n",
      "step1060:\n",
      "Train accuracy:  0.9192\n",
      "Test accuracy:  0.9196\n",
      "step1070:\n",
      "Train accuracy:  0.9168\n",
      "Test accuracy:  0.9172\n",
      "step1080:\n",
      "Train accuracy:  0.914\n",
      "Test accuracy:  0.9116\n",
      "step1090:\n",
      "Train accuracy:  0.9166\n",
      "Test accuracy:  0.919\n",
      "step1100:\n",
      "Train accuracy:  0.9198\n",
      "Test accuracy:  0.9184\n",
      "step1110:\n",
      "Train accuracy:  0.9208\n",
      "Test accuracy:  0.9114\n",
      "step1120:\n",
      "Train accuracy:  0.9244\n",
      "Test accuracy:  0.922\n",
      "step1130:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.918\n",
      "step1140:\n",
      "Train accuracy:  0.9148\n",
      "Test accuracy:  0.922\n",
      "step1150:\n",
      "Train accuracy:  0.9212\n",
      "Test accuracy:  0.9158\n",
      "step1160:\n",
      "Train accuracy:  0.9196\n",
      "Test accuracy:  0.9198\n",
      "step1170:\n",
      "Train accuracy:  0.921\n",
      "Test accuracy:  0.9142\n",
      "step1180:\n",
      "Train accuracy:  0.9196\n",
      "Test accuracy:  0.9206\n",
      "step1190:\n",
      "Train accuracy:  0.9188\n",
      "Test accuracy:  0.918\n",
      "step1200:\n",
      "Train accuracy:  0.9066\n",
      "Test accuracy:  0.9166\n",
      "step1210:\n",
      "Train accuracy:  0.9184\n",
      "Test accuracy:  0.9152\n",
      "step1220:\n",
      "Train accuracy:  0.9202\n",
      "Test accuracy:  0.9188\n",
      "step1230:\n",
      "Train accuracy:  0.92\n",
      "Test accuracy:  0.9214\n",
      "step1240:\n",
      "Train accuracy:  0.9218\n",
      "Test accuracy:  0.9154\n",
      "step1250:\n",
      "Train accuracy:  0.9206\n",
      "Test accuracy:  0.9194\n",
      "step1260:\n",
      "Train accuracy:  0.9012\n",
      "Test accuracy:  0.9124\n",
      "step1270:\n",
      "Train accuracy:  0.9182\n",
      "Test accuracy:  0.9184\n",
      "step1280:\n",
      "Train accuracy:  0.9228\n",
      "Test accuracy:  0.9216\n",
      "step1290:\n",
      "Train accuracy:  0.9204\n",
      "Test accuracy:  0.9214\n",
      "step1300:\n",
      "Train accuracy:  0.9094\n",
      "Test accuracy:  0.9112\n",
      "step1310:\n",
      "Train accuracy:  0.9262\n",
      "Test accuracy:  0.9234\n",
      "step1320:\n",
      "Train accuracy:  0.9182\n",
      "Test accuracy:  0.922\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step1330:\n",
      "Train accuracy:  0.9204\n",
      "Test accuracy:  0.9196\n",
      "step1340:\n",
      "Train accuracy:  0.9204\n",
      "Test accuracy:  0.9182\n",
      "step1350:\n",
      "Train accuracy:  0.9206\n",
      "Test accuracy:  0.9212\n",
      "step1360:\n",
      "Train accuracy:  0.9188\n",
      "Test accuracy:  0.9194\n",
      "step1370:\n",
      "Train accuracy:  0.9238\n",
      "Test accuracy:  0.9216\n",
      "step1380:\n",
      "Train accuracy:  0.9086\n",
      "Test accuracy:  0.9146\n",
      "step1390:\n",
      "Train accuracy:  0.9194\n",
      "Test accuracy:  0.9226\n",
      "step1400:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.9178\n",
      "step1410:\n",
      "Train accuracy:  0.9178\n",
      "Test accuracy:  0.9258\n",
      "step1420:\n",
      "Train accuracy:  0.915\n",
      "Test accuracy:  0.9158\n",
      "step1430:\n",
      "Train accuracy:  0.9226\n",
      "Test accuracy:  0.9214\n",
      "step1440:\n",
      "Train accuracy:  0.925\n",
      "Test accuracy:  0.9256\n",
      "step1450:\n",
      "Train accuracy:  0.9206\n",
      "Test accuracy:  0.9132\n",
      "step1460:\n",
      "Train accuracy:  0.9128\n",
      "Test accuracy:  0.9158\n",
      "step1470:\n",
      "Train accuracy:  0.9262\n",
      "Test accuracy:  0.923\n",
      "step1480:\n",
      "Train accuracy:  0.9242\n",
      "Test accuracy:  0.9222\n",
      "step1490:\n",
      "Train accuracy:  0.9214\n",
      "Test accuracy:  0.9248\n",
      "step1500:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9238\n",
      "step1510:\n",
      "Train accuracy:  0.9156\n",
      "Test accuracy:  0.9172\n",
      "step1520:\n",
      "Train accuracy:  0.921\n",
      "Test accuracy:  0.922\n",
      "step1530:\n",
      "Train accuracy:  0.9194\n",
      "Test accuracy:  0.925\n",
      "step1540:\n",
      "Train accuracy:  0.9204\n",
      "Test accuracy:  0.921\n",
      "step1550:\n",
      "Train accuracy:  0.919\n",
      "Test accuracy:  0.9252\n",
      "step1560:\n",
      "Train accuracy:  0.9288\n",
      "Test accuracy:  0.9192\n",
      "step1570:\n",
      "Train accuracy:  0.9244\n",
      "Test accuracy:  0.9234\n",
      "step1580:\n",
      "Train accuracy:  0.9244\n",
      "Test accuracy:  0.924\n",
      "step1590:\n",
      "Train accuracy:  0.927\n",
      "Test accuracy:  0.924\n",
      "step1600:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.924\n",
      "step1610:\n",
      "Train accuracy:  0.9114\n",
      "Test accuracy:  0.9214\n",
      "step1620:\n",
      "Train accuracy:  0.9286\n",
      "Test accuracy:  0.9268\n",
      "step1630:\n",
      "Train accuracy:  0.9312\n",
      "Test accuracy:  0.9196\n",
      "step1640:\n",
      "Train accuracy:  0.913\n",
      "Test accuracy:  0.917\n",
      "step1650:\n",
      "Train accuracy:  0.9212\n",
      "Test accuracy:  0.916\n",
      "step1660:\n",
      "Train accuracy:  0.9218\n",
      "Test accuracy:  0.9284\n",
      "step1670:\n",
      "Train accuracy:  0.92\n",
      "Test accuracy:  0.922\n",
      "step1680:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9236\n",
      "step1690:\n",
      "Train accuracy:  0.9238\n",
      "Test accuracy:  0.924\n",
      "step1700:\n",
      "Train accuracy:  0.9284\n",
      "Test accuracy:  0.9214\n",
      "step1710:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9262\n",
      "step1720:\n",
      "Train accuracy:  0.9216\n",
      "Test accuracy:  0.9226\n",
      "step1730:\n",
      "Train accuracy:  0.9236\n",
      "Test accuracy:  0.9232\n",
      "step1740:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.9254\n",
      "step1750:\n",
      "Train accuracy:  0.9122\n",
      "Test accuracy:  0.9132\n",
      "step1760:\n",
      "Train accuracy:  0.9132\n",
      "Test accuracy:  0.9184\n",
      "step1770:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9218\n",
      "step1780:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9248\n",
      "step1790:\n",
      "Train accuracy:  0.919\n",
      "Test accuracy:  0.921\n",
      "step1800:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9258\n",
      "step1810:\n",
      "Train accuracy:  0.913\n",
      "Test accuracy:  0.9124\n",
      "step1820:\n",
      "Train accuracy:  0.9234\n",
      "Test accuracy:  0.9224\n",
      "step1830:\n",
      "Train accuracy:  0.912\n",
      "Test accuracy:  0.9244\n",
      "step1840:\n",
      "Train accuracy:  0.9262\n",
      "Test accuracy:  0.9292\n",
      "step1850:\n",
      "Train accuracy:  0.9276\n",
      "Test accuracy:  0.9232\n",
      "step1860:\n",
      "Train accuracy:  0.9208\n",
      "Test accuracy:  0.9236\n",
      "step1870:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9274\n",
      "step1880:\n",
      "Train accuracy:  0.929\n",
      "Test accuracy:  0.9264\n",
      "step1890:\n",
      "Train accuracy:  0.923\n",
      "Test accuracy:  0.9226\n",
      "step1900:\n",
      "Train accuracy:  0.9216\n",
      "Test accuracy:  0.9242\n",
      "step1910:\n",
      "Train accuracy:  0.9226\n",
      "Test accuracy:  0.9244\n",
      "step1920:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9274\n",
      "step1930:\n",
      "Train accuracy:  0.9284\n",
      "Test accuracy:  0.9216\n",
      "step1940:\n",
      "Train accuracy:  0.9294\n",
      "Test accuracy:  0.9242\n",
      "step1950:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9248\n",
      "step1960:\n",
      "Train accuracy:  0.9266\n",
      "Test accuracy:  0.9258\n",
      "step1970:\n",
      "Train accuracy:  0.9252\n",
      "Test accuracy:  0.9234\n",
      "step1980:\n",
      "Train accuracy:  0.9194\n",
      "Test accuracy:  0.922\n",
      "step1990:\n",
      "Train accuracy:  0.9238\n",
      "Test accuracy:  0.9256\n",
      "step2000:\n",
      "Train accuracy:  0.9316\n",
      "Test accuracy:  0.9212\n",
      "step2010:\n",
      "Train accuracy:  0.9214\n",
      "Test accuracy:  0.9318\n",
      "step2020:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.921\n",
      "step2030:\n",
      "Train accuracy:  0.9268\n",
      "Test accuracy:  0.9236\n",
      "step2040:\n",
      "Train accuracy:  0.9268\n",
      "Test accuracy:  0.9274\n",
      "step2050:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9228\n",
      "step2060:\n",
      "Train accuracy:  0.925\n",
      "Test accuracy:  0.9286\n",
      "step2070:\n",
      "Train accuracy:  0.927\n",
      "Test accuracy:  0.9204\n",
      "step2080:\n",
      "Train accuracy:  0.9092\n",
      "Test accuracy:  0.9174\n",
      "step2090:\n",
      "Train accuracy:  0.9216\n",
      "Test accuracy:  0.925\n",
      "step2100:\n",
      "Train accuracy:  0.9282\n",
      "Test accuracy:  0.925\n",
      "step2110:\n",
      "Train accuracy:  0.9262\n",
      "Test accuracy:  0.9238\n",
      "step2120:\n",
      "Train accuracy:  0.929\n",
      "Test accuracy:  0.9244\n",
      "step2130:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9212\n",
      "step2140:\n",
      "Train accuracy:  0.9252\n",
      "Test accuracy:  0.9282\n",
      "step2150:\n",
      "Train accuracy:  0.9262\n",
      "Test accuracy:  0.9266\n",
      "step2160:\n",
      "Train accuracy:  0.9266\n",
      "Test accuracy:  0.9222\n",
      "step2170:\n",
      "Train accuracy:  0.927\n",
      "Test accuracy:  0.9256\n",
      "step2180:\n",
      "Train accuracy:  0.925\n",
      "Test accuracy:  0.9216\n",
      "step2190:\n",
      "Train accuracy:  0.9288\n",
      "Test accuracy:  0.9258\n",
      "step2200:\n",
      "Train accuracy:  0.9264\n",
      "Test accuracy:  0.9258\n",
      "step2210:\n",
      "Train accuracy:  0.9174\n",
      "Test accuracy:  0.9164\n",
      "step2220:\n",
      "Train accuracy:  0.9262\n",
      "Test accuracy:  0.923\n",
      "step2230:\n",
      "Train accuracy:  0.929\n",
      "Test accuracy:  0.9266\n",
      "step2240:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9226\n",
      "step2250:\n",
      "Train accuracy:  0.9288\n",
      "Test accuracy:  0.9252\n",
      "step2260:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9256\n",
      "step2270:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.9232\n",
      "step2280:\n",
      "Train accuracy:  0.926\n",
      "Test accuracy:  0.9286\n",
      "step2290:\n",
      "Train accuracy:  0.925\n",
      "Test accuracy:  0.9228\n",
      "step2300:\n",
      "Train accuracy:  0.9228\n",
      "Test accuracy:  0.9246\n",
      "step2310:\n",
      "Train accuracy:  0.9244\n",
      "Test accuracy:  0.9248\n",
      "step2320:\n",
      "Train accuracy:  0.9226\n",
      "Test accuracy:  0.9248\n",
      "step2330:\n",
      "Train accuracy:  0.919\n",
      "Test accuracy:  0.9218\n",
      "step2340:\n",
      "Train accuracy:  0.9348\n",
      "Test accuracy:  0.9294\n",
      "step2350:\n",
      "Train accuracy:  0.923\n",
      "Test accuracy:  0.9198\n",
      "step2360:\n",
      "Train accuracy:  0.9116\n",
      "Test accuracy:  0.9148\n",
      "step2370:\n",
      "Train accuracy:  0.926\n",
      "Test accuracy:  0.9236\n",
      "step2380:\n",
      "Train accuracy:  0.926\n",
      "Test accuracy:  0.924\n",
      "step2390:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9246\n",
      "step2400:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9238\n",
      "step2410:\n",
      "Train accuracy:  0.9226\n",
      "Test accuracy:  0.9262\n",
      "step2420:\n",
      "Train accuracy:  0.9252\n",
      "Test accuracy:  0.923\n",
      "step2430:\n",
      "Train accuracy:  0.9242\n",
      "Test accuracy:  0.9302\n",
      "step2440:\n",
      "Train accuracy:  0.927\n",
      "Test accuracy:  0.9218\n",
      "step2450:\n",
      "Train accuracy:  0.9156\n",
      "Test accuracy:  0.9176\n",
      "step2460:\n",
      "Train accuracy:  0.932\n",
      "Test accuracy:  0.93\n",
      "step2470:\n",
      "Train accuracy:  0.9326\n",
      "Test accuracy:  0.9246\n",
      "step2480:\n",
      "Train accuracy:  0.9206\n",
      "Test accuracy:  0.9264\n",
      "step2490:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9256\n",
      "step2500:\n",
      "Train accuracy:  0.9264\n",
      "Test accuracy:  0.9218\n",
      "step2510:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.9284\n",
      "step2520:\n",
      "Train accuracy:  0.9264\n",
      "Test accuracy:  0.9214\n",
      "step2530:\n",
      "Train accuracy:  0.924\n",
      "Test accuracy:  0.9252\n",
      "step2540:\n",
      "Train accuracy:  0.912\n",
      "Test accuracy:  0.9146\n",
      "step2550:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9182\n",
      "step2560:\n",
      "Train accuracy:  0.9246\n",
      "Test accuracy:  0.9278\n",
      "step2570:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9286\n",
      "step2580:\n",
      "Train accuracy:  0.9232\n",
      "Test accuracy:  0.9218\n",
      "step2590:\n",
      "Train accuracy:  0.92\n",
      "Test accuracy:  0.9266\n",
      "step2600:\n",
      "Train accuracy:  0.9222\n",
      "Test accuracy:  0.9214\n",
      "step2610:\n",
      "Train accuracy:  0.9172\n",
      "Test accuracy:  0.9202\n",
      "step2620:\n",
      "Train accuracy:  0.923\n",
      "Test accuracy:  0.9214\n",
      "step2630:\n",
      "Train accuracy:  0.9242\n",
      "Test accuracy:  0.9258\n",
      "step2640:\n",
      "Train accuracy:  0.9232\n",
      "Test accuracy:  0.9298\n",
      "step2650:\n",
      "Train accuracy:  0.9234\n",
      "Test accuracy:  0.9208\n",
      "step2660:\n",
      "Train accuracy:  0.9222\n",
      "Test accuracy:  0.9242\n",
      "step2670:\n",
      "Train accuracy:  0.9274\n",
      "Test accuracy:  0.9238\n",
      "step2680:\n",
      "Train accuracy:  0.9226\n",
      "Test accuracy:  0.9292\n",
      "step2690:\n",
      "Train accuracy:  0.9274\n",
      "Test accuracy:  0.9238\n",
      "step2700:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9278\n",
      "step2710:\n",
      "Train accuracy:  0.9256\n",
      "Test accuracy:  0.9214\n",
      "step2720:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9222\n",
      "step2730:\n",
      "Train accuracy:  0.9242\n",
      "Test accuracy:  0.9264\n",
      "step2740:\n",
      "Train accuracy:  0.922\n",
      "Test accuracy:  0.9268\n",
      "step2750:\n",
      "Train accuracy:  0.9228\n",
      "Test accuracy:  0.9266\n",
      "step2760:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.921\n",
      "step2770:\n",
      "Train accuracy:  0.936\n",
      "Test accuracy:  0.9268\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step2780:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9254\n",
      "step2790:\n",
      "Train accuracy:  0.9258\n",
      "Test accuracy:  0.9266\n",
      "step2800:\n",
      "Train accuracy:  0.9266\n",
      "Test accuracy:  0.9212\n",
      "step2810:\n",
      "Train accuracy:  0.925\n",
      "Test accuracy:  0.926\n",
      "step2820:\n",
      "Train accuracy:  0.9224\n",
      "Test accuracy:  0.9248\n",
      "step2830:\n",
      "Train accuracy:  0.9114\n",
      "Test accuracy:  0.92\n",
      "step2840:\n",
      "Train accuracy:  0.928\n",
      "Test accuracy:  0.9242\n",
      "step2850:\n",
      "Train accuracy:  0.9286\n",
      "Test accuracy:  0.9228\n",
      "step2860:\n",
      "Train accuracy:  0.9268\n",
      "Test accuracy:  0.926\n",
      "step2870:\n",
      "Train accuracy:  0.9286\n",
      "Test accuracy:  0.9236\n",
      "step2880:\n",
      "Train accuracy:  0.9252\n",
      "Test accuracy:  0.928\n",
      "step2890:\n",
      "Train accuracy:  0.9248\n",
      "Test accuracy:  0.9254\n",
      "step2900:\n",
      "Train accuracy:  0.9178\n",
      "Test accuracy:  0.9166\n",
      "step2910:\n",
      "Train accuracy:  0.9162\n",
      "Test accuracy:  0.9212\n",
      "step2920:\n",
      "Train accuracy:  0.9208\n",
      "Test accuracy:  0.9254\n",
      "step2930:\n",
      "Train accuracy:  0.9274\n",
      "Test accuracy:  0.9242\n",
      "step2940:\n",
      "Train accuracy:  0.9272\n",
      "Test accuracy:  0.922\n",
      "step2950:\n",
      "Train accuracy:  0.9286\n",
      "Test accuracy:  0.9278\n",
      "step2960:\n",
      "Train accuracy:  0.926\n",
      "Test accuracy:  0.9214\n",
      "step2970:\n",
      "Train accuracy:  0.9274\n",
      "Test accuracy:  0.9304\n",
      "step2980:\n",
      "Train accuracy:  0.9224\n",
      "Test accuracy:  0.9214\n",
      "step2990:\n",
      "Train accuracy:  0.9264\n",
      "Test accuracy:  0.9282\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import Batch as bh\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "pathdataSet = 'D:/homework/pre/dataSetSamplerfilterFeature.csv'\n",
    "pathSaveModel = 'D:/homework/pre/net/mynet'\n",
    "pathTrainLogs = 'D:/homework/pre/logs/train'\n",
    "pathTestLogs = 'D:/homework/pre/logs/test'\n",
    "\n",
    "#划分训练集和测试集比例\n",
    "testSize = 0.4\n",
    "\n",
    "dataSet = pd.read_csv(pathdataSet)\n",
    "dataSet = shuffle(dataSet)\n",
    "dataTrain , dataTest = train_test_split(dataSet, test_size =testSize)\n",
    "\n",
    "m,n = dataTrain.shape\n",
    "dataTrain = bh.Dataset(dataTrain)\n",
    "dataTest = bh.Dataset(dataTest)\n",
    "\n",
    "\n",
    "#模型参数设置\n",
    "lr=0.01\n",
    "#这里不要超过test的最大样本数数\n",
    "batch_size=5000               #每一类大概抽取50个样本\n",
    "training_iters = 3000         #最大迭代次数\n",
    "n_inputs= n-1                #每个inputs大概有28个特征\n",
    "n_steps = 1                   #这里没有涉及到时间序列\n",
    "# n_inputs = 1\n",
    "# n_steps = n-1\n",
    "n_hidden_units=20           #隐藏层个数\n",
    "n_classes=12                #类别数\n",
    "\n",
    "\n",
    "with tf.name_scope('Input'):\n",
    "    x=tf.placeholder(tf.float32,[None,n_steps,n_inputs],name=\"x\")\n",
    "    y=tf.placeholder(tf.float32,[None],name=\"y\")\n",
    "#同时定义输入和输出的weights和biases\n",
    "weights={\n",
    "        'in':tf.Variable(tf.random_normal([n_inputs,n_hidden_units])),\n",
    "        'out':tf.Variable(tf.random_normal([n_hidden_units,n_classes]))\n",
    "        }\n",
    "biases={\n",
    "        'in':tf.Variable(tf.constant(0.1,shape=[n_hidden_units,])),\n",
    "        'out':tf.Variable(tf.constant(0.1,shape=[n_classes,]))\n",
    "        }\n",
    "\n",
    "y_one_hot = tf.one_hot(tf.cast(y,tf.int32),n_classes,on_value=1,off_value=None,axis=1)\n",
    "\n",
    "with tf.name_scope('LstmModel'):\n",
    "    def RNN(X,weights,biases):\n",
    "        X=tf.reshape(X,[-1,n_inputs])\n",
    "        X_in=tf.matmul(X,weights['in'])+biases['in']\n",
    "        X_in=tf.reshape(X_in,[-1,n_steps,n_hidden_units])\n",
    "        #使用basicLSTMcell，n_hidden表示神经元的个数，forget_bias就是LSTM们的忘记系数，如果等于1，\n",
    "        #就是不会忘记任何信息。如果等于0，就都忘记。state_is_tuple使用True，就是表示返回的状态用一个元组表示，\n",
    "        #对lstm来说，返回的状态是一个包含两个元素的元组：(c_state,h_state) \n",
    "        lstm_cell=tf.nn.rnn_cell.BasicLSTMCell(n_hidden_units,forget_bias=1.0,state_is_tuple=True)\n",
    "        #初始化全零state，batch_size就是输入样本批次的数目，dtype就是数据类型。\n",
    "        _init_state=lstm_cell.zero_state(batch_size,dtype=tf.float32)\n",
    "        #使用tf.nn.dynamic_rnn(cell, inputs)要确定inputs的格式，time_major参数会针对不同inputs格式有不同的值.\n",
    "        #1.如果 inputs 为 (batches, steps, inputs) ==> time_major=False;\n",
    "        #2.如果 inputs 为 (steps, batches, inputs) ==> time_major=True;\n",
    "        #我们这里是用的第一种（看placeholder，steps在第二个，所以要设为False\n",
    "        outputs,states=tf.nn.dynamic_rnn(lstm_cell,X_in,initial_state=_init_state,time_major=False)\n",
    "        results=tf.matmul(states[1],weights['out']+biases['out'])\n",
    "        return results\n",
    "\n",
    "\n",
    "pred=RNN(x,weights,biases)\n",
    "\n",
    "with tf.name_scope('loss'):\n",
    "    cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_one_hot,logits=pred))\n",
    "    tf.summary.scalar('loss',cost)\n",
    "\n",
    "with tf.name_scope('train_step'):\n",
    "    train_op=tf.train.AdamOptimizer(lr).minimize(cost)\n",
    "    \n",
    "with tf.name_scope('Accuracy'):\n",
    "    correct_pred=tf.equal(tf.argmax(pred,1),tf.argmax(y_one_hot,1))\n",
    "    accuracy=tf.reduce_mean(tf.cast(correct_pred,tf.float32),name=\"Accuracy\")\n",
    "    tf.summary.scalar('Accuracy',accuracy)\n",
    "\n",
    "init=tf.global_variables_initializer()\n",
    "merged=tf.summary.merge_all()\n",
    "saver=tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    writer_train = tf.summary.FileWriter(pathTrainLogs,sess.graph)\n",
    "    writer_test = tf.summary.FileWriter(pathTestLogs,sess.graph)\n",
    "\n",
    "   \n",
    "    \n",
    "    step=0\n",
    "    while step<training_iters:\n",
    "        dataSet = dataTrain.next_batch(batch_size)\n",
    "        batch_train_xs = np.array(dataSet.drop(['label'],axis = 1))\n",
    "        batch_train_xs = batch_train_xs.reshape([-1,n_steps,n_inputs])\n",
    "        batch_train_ys = np.array(dataSet['label'])\n",
    "        sess.run([train_op],feed_dict = {x:batch_train_xs,y:batch_train_ys})\n",
    "        \n",
    "        if step%10==0:\n",
    "            print('step'+str(step)+':')\n",
    "            print('Train accuracy: ',sess.run(accuracy,feed_dict={x:batch_train_xs,y:batch_train_ys}))\n",
    "            summary=sess.run(merged,feed_dict={x: batch_train_xs,y:batch_train_ys})\n",
    "            writer_train.add_summary(summary,step)\n",
    "            \n",
    "            \n",
    "            dataSet = dataTest.next_batch(batch_size)\n",
    "            batch_test_xs = np.array(dataSet.drop(['label'],axis = 1))\n",
    "            batch_test_xs = batch_test_xs.reshape([-1,n_steps,n_inputs])\n",
    "            batch_test_ys = np.array(dataSet['label'])\n",
    "            \n",
    "            print('Test accuracy: ',sess.run(accuracy,feed_dict = {x: batch_test_xs,y: batch_test_ys}))\n",
    "            summary=sess.run(merged,feed_dict = {x:batch_test_xs,y:batch_test_ys})\n",
    "            writer_test.add_summary(summary,step)\n",
    "        step+=1    \n",
    "\n",
    "    saver.save(sess,pathSaveModel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
